#coding:utf-8
from BloomFilter import BloomFilter
from spider.core.baseSpider import baseSpider

class bloomFilterSpider(baseSpider):
    """ 添加bloom过滤器的爬虫 """
    def __init__(self,bloom,table):
        baseSpider.__init__(self,table)
        if bloom:
            self.bloom = bloom
        else:
            self.bloom = BloomFilter(0.1,1000)
    def existUrl(self,url):
        exist = self.bloom.exists(url)
        if not exist:
            self.bloom.insert(url)
        return exist
    def saveData(self):
        # 析构时保存数据
        self.bloom.saveData()